"""
This script is used to run all the test of openttd performance
"""
import os,time, NetworkCommunication, string
from Util import *
from threading import Timer
from Performance import Performance
from Configuration import Configuration
import threading, math, sys, DescriptionParser, FaultTolerance,socket, SubmitNewExps, thread, JobExecution
#ft = True
ft = False
Configuration = Configuration()
instanceOfPrun = 1
totalPrunLimit = 1
expNames = [];
experiments = [];
#expNames = ["1a1-1024", "1a1-2048", "1a4", "1a1-64", "1a1-128", "1a1-256"]
#expNames = ["1a7",
#            "1a11_0_0", "1a11_0_1", "1a11_0_2", "1a11_0_3",
#            "1a11_0_0", "1a11_0_1", "1a11_0_2", "1a11_0_3", "1a12", "1a12", "1a12"] 
#            "1a11_3_0", "1a11_3_1", "1a11_3_2", "1a11_3_3", "1a7"]
#expNames = ["1a5"]
#expNames = ["1a13-6","1a13-6", "1a13-6", "quick-1a14-5y", "quick-1a14-5y","quick-1a14-5y","quick-1a15-5y", "quick-1a15-5y", "quick-1a15-5y"]
#expNames = ["1a13-6","quick-1a11_0_0-5y", "quick-1a11_0_1-5y","quick-1a11_0_2-5y","quick-1a11_0_3-5y"
#            ,"quick-1a11_0_0-5y", "quick-1a11_0_1-5y","quick-1a11_0_2-5y","quick-1a11_0_3-5y"
#            ,"quick-1a11_0_0-5y", "quick-1a11_0_1-5y","quick-1a11_0_2-5y","quick-1a11_0_3-5y"]
#expNames = ["1a7-1024", "1a7-2048", "1a7-1024", "1a7-2048", "1a7-1024","1a7-2048" ]
#expNames = ["quick-1a11_0_0-5y"];
#expNames = ["quick-1a14-5y", "quick-1a14-5y", "quick-1a14-5y", "quick-1a15-5y", "quick-1a15-5y", "quick-1a15-5y"]
#expNames = ["quick-1a8-5y","quick-1a8-5y","quick-1a8-5y"]
#expNames = ["quick-1a15-1y"]
#experiments=[]
#experiments = [["1a7-1024-op40000", "anyServer", "1", 100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160]]
#experiments = [["1a12", "anyServer", "10", 3,5,13]]
#experiments = [["2a7-desktop", "anyServer", "1", 100]]

#minVars = { "1a1":1, "1a2":6, "1a3":1, "1a4":1, "1c":1, "1a6-1024":1, 
#           "1a6-2048":1, "1a6-256":1,  "1a5":1, "1a6":1,
#           "2a6":50, "3a6":16, "2a6-2048":16, "2a7":16, "1a7":94}
#maxVars = { "1a1":16, "1a2":11, "1a3":4, "1a4":1, "1c":4, 
#           "1a6-1024":16, "1a6-2048":16, "1a6-256":4, "1a5":17, "1a6":16,
#           "2a6":100, "3a6":50, "2a6-2048":100, "2a7":200, "1a7":160}

#differnt cluster has different configuration.....................
servers = [
           "fs3.das4.tudelft.nl"
#           ,"fs0.das4.cs.vu.nl"
#           ,
#           "fs1.das3.liacs.nl",
#           "fs2.das3.science.uva.nl",
           
#           , "fs4.das3.science.uva.nl"
           ]


#def allocateStatus(expName, factor):
#    i = 0
#    for k in minVars.keys():
#        if k == expName:
#            break;
#        i = i + 1
#    rank = 0
#    for j in (0, i):
#        name = minVars.keys()[j]
#        rank = rank + (maxVars[name] - minVars[name] + 1)
#    rank = rank + factor
#    return rank

def isStillSomeExpInWaiting(servername, expName, factor, hostname="fs3"):
    cmd =  "qstat |grep " + Configuration.userName;
#    print hostname
    if servername.find(hostname) <0:
        cmd = "ssh -i "+ Configuration.sshKey + " " + Configuration.userName + "@"+ servername + " '" + cmd + "'";
#    print cmd
    file = os.popen(cmd)
    lines = file.readlines()
    foundWaiting = False
    totalRun = 0
    if len(lines) == 0:
#        print "len(lines)==0 return False"
        return False
    for line in lines:
        tup = string.split(line)
#        print tup
        if tup[4] !="r":
#            print "foundWaiting"
            return True
        if tup[4] == "r":
#            print "totalRun =", totalRun +1
            totalRun = totalRun + 1
    if foundWaiting == True:
        return True
    if totalRun >= totalPrunLimit :
        return True

#    print "There is still machine to run on ", servername
    return False;

def getDelay(delay):
    if delay >0 :
        return "-delay %d "%delay
    else: return ""

"""
This module will determine whether the experiment will be ran in multi-cluster or 
"""
def runExpInServer(servername, expId, expName, maxYear, factor, instances=16,delay = 0, permachine=1):
    print "-------------------------------------------------"
    ttt = Configuration.getTimelimit(maxYear, servername)
    ddd = getDelay(delay)
    numberOfProcess = "-%d "%permachine
    cmd = "prun -v -np %d "%instances + ddd + numberOfProcess + ttt + "python " + Configuration.remoteScriptDirectory + "JobExecution.py" + " %d"%expId + " " + expName + " " + "%d"%maxYear + " " + "%d"%factor

    print cmd
#    servername = findServer(expName, factor)
    if ft == True:
        cmd = cmd + " nullPath ft"
    hostname = Configuration.getHostName()
    cmd = "cd " + Configuration.remoteScriptDirectory + ";" + cmd;
    if servername.find(hostname)<0:
        cmd = "ssh -i "+ Configuration.sshKey + " " + Configuration.userName + "@"+ servername + " '" + cmd + "'"
    print cmd
    "Might have some connection problems here, but I have never encounter any network problem in DAS-4, in Cloud, there may be some problems"
    os.system(cmd)
    print "-----------------------------------------------------"
    

def runInLocal(expId, expName, maxYear, factor):
    print "--- running in local -------------------"
    print expId, expName, maxYear, factor;
    if ft == True:
        JobExecution.testSingle(expId, expName, maxYear, factor, "nonull", "ft")
    else :
        JobExecution.testSingle(expId, expName, maxYear, factor, "nonull")
    
    
def runInMyComputer(expId, expName, maxYear, factor):
    print "-------------------------------------------------"
    hostname = Configuration.getHostName();
    if hostname != Configuration.desktopHostname:    
        servername = Configuration.desktopUserName+"@" + Configuration.desktopIPAddress
#        filename = "../openttd.cfg"
#        cmd = "scp -i " + Configuration.sshKey + " " + filename + " " + servername + ":~/.openttd/&"
#        print cmd
#        os.system(cmd)
#        filename = "../game_start.scr"
#        cmd = "scp -i " + Configuration.sshKey + " " + filename + " " + servername + ":~/1.0.5_modified/bin/scripts/&"
#        print cmd
#        os.system(cmd)
        cmd = "cd " + Configuration.remoteScriptDirectory + "; python JobExecution.py " + "%d"%expId + " " + expName + " " + "%d"%maxYear + " " + "%d"%factor + " NONULL"
        if ft == True:
            cmd = cmd + " " + "ft"
        cmd = "ssh -i " + Configuration.sshKey + " " + servername + " '" + cmd + "&'&"
        print cmd
        os.system(cmd)
    elif hostname.find(Configuration.desktopHostname) >=0: 
        """Local computer"""
        if ft == True:
            JobExecution.testSingle(expId, expName, maxYear, factor, "nonull", "ft")
        else :
            JobExecution.testSingle(expId, expName, maxYear, factor, "nonull")
    print "-----------------------------------------------------"
    return True


#
#def runServer(expId, expName, maxYear, factor):
#    runExpInServer(expId, expName, maxYear, factor, 1)
#    Configuration.setAI(0)
#    return True
#
#def runClient(expId, expName, maxYear, factor):
#    expName = expName[1:];
#    expName = "3" +expName;
#    Configuration.setAI(0)
#    if expName.find("3a6") >=0:
#        runExpInServer(expId, expName, maxYear, factor, factor, 4)
#    elif expName.find("3a2") >=0:
#        runExpInServer(expId, expName, maxYear, factor, 8, 4)
#    return True

def removeAutoSaveOnServer(servername):
    return


def linuxPathJoin(path1, path2):
    return path1 + "/" + path2;

def copyFileStub(expName, template,servername):
    src = Configuration.getPathLinux();
    if(src[len(src)-1]=='/'):
        src = src[0:len(src)-1];
    linuxSrc = src + "";
#    dest = rp(os.path.join(rt(Configuration.copyToDirectory),expName));
    dest = linuxPathJoin(Configuration.copyToDirectory, expName);
    dest = Configuration.copyToDirectory + "/" + expName;
    mkdir(dest);
    cmd = "scp -i " + Configuration.sshKey + " -r " + Configuration.userName + "@" + servername + ":"+ src + "/" + expName + "/" + template + "* " + dest;
    print cmd
    os.system(cmd)
    
    dest = Configuration.copyToDirectory + "/" + expName + "/" + "analysis";
    mkdir(dest);
    src = linuxSrc + "/" + expName + "/analysis/" + template;
    cmd = "scp -i " + Configuration.sshKey + " -r " + Configuration.userName + "@" + servername + ":"+ src  + "* " + dest;
    print cmd
    os.system(cmd)
    
    dest = Configuration.copyToDirectory + "/" + expName + "/" + "autosave";
    mkdir(dest);
    src = linuxSrc + "/" + expName + "/autosave/" + template;
    cmd = "scp -i " + Configuration.sshKey + " -r " + Configuration.userName + "@" + servername + ":"+ src + "* " + dest;
    print cmd
    os.system(cmd)
    
    dest = Configuration.copyToDirectory + "/" + expName + "/" + "network";
    mkdir(dest);
    src = linuxSrc + "/" + expName + "/network/" + template;
    cmd = "scp -i " + Configuration.sshKey + " -r " + Configuration.userName + "@" + servername + ":"+ src + "* " + dest;
    print cmd
    os.system(cmd)

def copyAndDelResultFromDas3(expId, expName, factor, servername):
    template = "%d_"%expId + expName + "_%d_"%factor
    tPath = Configuration.getPathLinux();
    if(tPath[len(tPath)-1]=='/'):
        tPath = tPath[0:len(tPath)-1];
    tPath = toLinux(tPath);
    cmd = "scp -i " + Configuration.sshKey + " -r " + Configuration.userName + "@" + servername + ":"+ tPath + "/" + template + "* " + toCygwin(Configuration.copyToDirectory);
    print cmd
    os.system(cmd)
    copyFileStub(expName, template,servername);
    """This function is called by scheduler"""
    preFixcmd = "ssh -i " + Configuration.sshKey + " " + Configuration.userName + "@"+servername;
    cmd = preFixcmd + " '" + "rm -fr " + tPath + "/" + template + "*" + "'";
    print cmd
    os.system(cmd)
    cmd = preFixcmd + " '" + "rm -fr " + tPath + "/" + expName + "/analysis/*" + "'";
    print cmd
    os.system(cmd)

    cmd = preFixcmd + " '" + "rm -fr " + tPath + "/" + expName + "/autosave/*" + "'";
    print cmd
    os.system(cmd)
    return True

def changeConfigurationAndSendtoServer(servername, expName, factor):
    hostname = Configuration.getHostName()
#    Configuration.initExp(expName)
#    Configuration.expVarChange(expName, factor)
    server = servername
#    copyfile(expName, server, hostname)
    time.sleep(2)
    removeAutoSaveOnServer(server)
    return True

def getServer(expName, factor):
    hostname = Configuration.getHostName()
    while True:
        for servername in servers:
            if isStillSomeExpInWaiting(servername, expName, factor, hostname) == False:
                print servername, "is Idle"
                return servername
        time.sleep(10)

def realRunExpriment(expId, expName, i, servername, maxYear, numberOfInstance=instanceOfPrun):
    print "starting experiment on ", expId, expName, "factor", i, servername, "maxYear", maxYear
#    changeConfigurationAndSendtoServer(servername, expName, i)
#    removeAutoSaveOnServer(servername)
    hostname = Configuration.getHostName()
    cpuPerNode = 8
    if hostname.find("das3")>=0:
        cpuPerNode = 4
#            numberOfMachine = min(16,i)
#            if numberOfMachine < 16:
#                numberOfMachine = numberOfMachine + 1;
    numberOfMachine = i;
    """
    Only test expName starts with "1a"; for expName starts with 2, 3, not tested yet
    """
    if expName[0:1] == "2":
        """
        start registry Server for registering app client/server
        """
        NetworkCommunication.start_server(numberOfMachine+1, i)            
        if expName.find("desktop") >= 0:
            numberOfMachine = numberOfMachine - 1
        delay = 2
        runExpInServer(servername, expId, expName, maxYear, i, 1, delay) #openttdserver
        tt = numberOfMachine//cpuPerNode
        time.sleep(10)
        if expName.find("desktop") >= 0:
            time.sleep(20)
        """
        In job execution module, it will connect to registry server to decide whether this instance is app server/client
        """
        if tt >0:
            runExpInServer(servername, expId, expName, maxYear, i, tt, delay, cpuPerNode)
        if numberOfMachine%cpuPerNode !=0:
            time.sleep(delay * numberOfMachine + delay)
            runExpInServer(servername, expId, expName, maxYear, i, 1, delay, numberOfMachine%cpuPerNode)
        time.sleep(10)
        
        if expName.find("desktop") >=0:
            runInMyComputer(expId, expName, maxYear, i)
        time.sleep(300 * maxYear)
        NetworkCommunication.s.close()
        NetworkCommunication.enableRunFlag = False
    elif expName.find("4a")>=0:
        runInMyComputer(expId, expName, maxYear, i)
        time.sleep(5)
        runExpInServer(servername, expId, expName, maxYear, i, i, 2)
        time.sleep(300 * maxYear)
    elif expName.find("5a")>=0:
        NetworkCommunication.start_server(numberOfMachine+1, i)            
        delay = 2
        runExpInServer(servername, expId, expName, maxYear, i, 1, delay) #openttdserver
        numberOfMachine = numberOfMachine -1
        tt = numberOfMachine//8
        runExpInServer(servername, expId, expName, maxYear, i, tt, delay, 8)
    
        if numberOfMachine%8 !=0:
            time.sleep(delay * numberOfMachine + delay)
            runExpInServer(servername, expId, expName, maxYear, i, 1, delay, numberOfMachine%8)
        runInMyComputer(expId, expName, maxYear)
        time.sleep(300 * maxYear)
        NetworkCommunication.s.close()
        NetworkCommunication.enableRunFlag = False  
    elif expName.find("1a") >=0:
        print "The game instance will be schedule to run %d year(s)"%maxYear
        runExpInServer(servername, expId, expName, maxYear, i, numberOfInstance, 1)
        t = Configuration.getSleepTime(maxYear, servername)
        t = 10;
        """should write a process to check when this experiment is finished, if finished then copy"""
        time.sleep(t)
#        num = FaultTolerance.getErrorNum(expName, i, instanceOfPrun)
#                import sys
#                sys.stdin.read()
        """fault tolerance module should be further tested"""
        hostname = Configuration.getHostName()
        if ft == True and (hostname == "fs3" or hostname == "fs3.das4.tudelft.nl"):
            rep = 5
            while(rep > 0):
                print "begin fault tolerance " + time.asctime()
                num = FaultTolerance.getErrorNum(expName, i, numberOfInstance)
                print "expect number of experiment result %d" %numberOfInstance
                print "incorrect is %d"%num
                if num > 0:
                    expId = Configuration.increaseId()
                    print "begin fault tolerance run " + time.asctime()
                    SubmitNewExps.submitExps(expName, [i], num, servername)
                elif num == 0:
                    break
                time.sleep(t)
                rep = rep - 1
    if(servername.find(hostname)<0):
        copyAndDelResultFromDas3(expId, expName, i, servername)
    return True
def insertExpIntoExperiments():
    global experiments
    for expName in expNames:
        l1 = DescriptionParser.getFactors(expName)
        l = [expName]
        l.append("anyServer")
        l.append("%d"%instanceOfPrun)
        l.extend(l1)
        experiments.append(l)
        l = []
    
def listenNewJobs():
    listenerReady = False;
    try:
        global experiments
        s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
        print "---------------------------------------------------"
        print "the port is ", SubmitNewExps.port
        print "-----------------------------------------------------"
        s.bind(("",SubmitNewExps.port))
        s.listen(50)
        listenerReady = False;
        while(True):
            con,addr = s.accept();
            con.settimeout(10)
            str = con.recv(1024)
            con.send("ok")
            print "receive new exps " + str
            l = list(string.split(str))
            expName = l[0]
            servername = l[1]
            numberOfInstance = l[2]
            sfactors = l[3:]
        
            factors = []
            nl = [expName, servername, numberOfInstance]
            for sfactor in sfactors:
                factors.append(string.atoi(sfactor))
            nl.extend(factors)
            experiments.append(nl)
    #        if len(list) == 1:
    #            con.send("server")
    #            serveraddress = str
            print "listenNewJobs", experiments
            con.close()
        s.close()
    finally:
        if(listenerReady == False):
            sys.stderr.write("Port for listening incoming experiments is occupied\n");
            thread.interrupt_main();
            #sys.exit(0);quit(); do not work
        s.close()
    return True


def testAll():
    global expNames
    global experiments
    print expNames
    insertExpIntoExperiments()
    print experiments
    tid = thread.start_new_thread(listenNewJobs, ())
    while True:
#        print "sleeping, ", experiments
        if len(experiments) ==0:
            time.sleep(5)
#            threadlist = threading.enumerate()
#            if(len(threadlist) == 1): 
##                only the main thread is active, sub-thread is exited cause by occupied port
#                quit();
#            else:
            print "waiting for experiments"
        else:
            print "starting experiment"
            exp = experiments.pop(0) #if experiments.pop() is stack, not queue
            expName = exp[0]
            reqservername = exp[1:2][0]
            numOfInstance = string.atoi(exp[2:3][0])
            factors = exp[3:]
            Configuration.increaseId()
            expId = Configuration.getId(); 
            maxYear = Configuration.getMaxYear(expName)
            print "test ", factors
            for factor in factors:
#                print expName, factor, "run in ", servername
                if reqservername == "anyServer":
                    print "This exp is allowed to run in anyServer"
                    servername = getServer(expName, factor)
                    print "Scheduling to run in " + servername
                    thread.start_new_thread(realRunExpriment,(expId, expName, factor, servername, maxYear, numOfInstance))
                elif reqservername == "local":
                    runInLocal(expId, expName, maxYear, factor);
                else:
                    servername = reqservername;
                    print "This exp requires server : " + servername
                    while isStillSomeExpInWaiting(servername, expName, factor):
                        time.sleep(10)
                    thread.start_new_thread(realRunExpriment,(expId, expName, factor, servername, maxYear, numOfInstance))
                time.sleep(15)
        time.sleep(1)
#    for expName in expNames:
#        Configuration.increaseId()
#        expId = Configuration.getId();    
#        maxYear = Configuration.getMaxYear(expName)
#        print DescriptionParser.getFactors(expName)
#        for i in DescriptionParser.getFactors(expName):
#            servername = getServer(expName, i)
#            print "found a server " + servername
#            thread.start_new_thread(realRunExpriment,(expId, expName, i, servername, maxYear))
#            time.sleep(30)
if __name__ == "__main__":
#    mkdir("/cygdrive/c/t/123")
    testAll()
